﻿#include <iostream>
#include <string>
#include <fstream>
#include <cassert>
//#include <cstdlib>
#include <vector>
#include <cmath>
//#include <string.h>
//#include <stdio.h>
//#include <iconv.h>
#include "errno.h"
#include "cppjieba/include/cppjieba/Jieba.hpp"
#include <map>
#include <set>
#include "eigen/Eigen/Dense"  
#define judgeletter_bit(ch) (((ch)>>6)==1)&&((((ch)-1)&31)<26) 
#include "CScipyDistance.h"
#include "CEditDistance.h"

using namespace Eigen; 
using namespace std;
namespace text_match
{
float russellrao(VectorXf u, VectorXf v);
void count_not_bool(VectorXf u, VectorXf v, float &tt, float &tf, float &ff, float &ft);
float braycurtis(VectorXf u, VectorXf v);
float sokalmichener(VectorXf u, VectorXf v);
float canberra(VectorXf u, VectorXf v);
float chebyshev(VectorXf u, VectorXf v);
float skew(VectorXf u);
float kurtosis(VectorXf u);

float getWeight(const vector<float> &weight, const int key, const vector<bool> &stop_words);
float wordMatch(vector<int> u, vector<int> v, const std::vector<bool> &stop_words, const std::vector<float> &weight);
void removeSortRepeat(vector<int> &u);

bool is_chinese(const string& str);
void split_string(const string& s, const string& delim,vector<string >& ret);
void Char2Token(const char* szSent, vector<string>& tokens);	


class TrainTemp {
public:
	vector<vector<string> > text1fenci_column;
	vector<vector<string> > text1fenzi_column;
	vector<vector<string> > text2fenci_column;
	vector<vector<string> > text2fenzi_column;
};

class PredictTemp {
public:
	vector<vector<string> > text1fenci_column_predict;
	vector<vector<string> > text1fenzi_column_predict;
	vector<vector<string> > text2fenci_column_predict;
	vector<vector<string> > text2fenzi_column_predict;
};

class FeatureResult {
public:
    vector<float> word_q2_kur;
    vector<float> char_q2_kur;
    vector<float> word_q1_kur;
    vector<float> char_q1_kur;
    vector<float> word_q2_skew;
    vector<float> char_q2_skew;
    vector<float> word_q1_skew;
    vector<float> char_q1_skew;
	
    vector<float> word_vec_mean_russellrao;
    vector<float> char_vec_mean_russellrao;
	vector<float> word_vec_mean_kulsinski;
    vector<float> char_vec_mean_kulsinski;
	vector<float> word_vec_mean_russellrao_norm;
    vector<float> char_vec_mean_russellrao_norm;
	vector<float> word_vec_mean_kulsinski_norm;
    vector<float> char_vec_mean_kulsinski_norm;
	vector<float> tfidf_word_yule;
	vector<float> tfidf_word_canberra;
	vector<float> tfidf_word_chebyshev;
	vector<float> tfidf_word_braycurtis;
	vector<float> tfidf_word_sokalmichener;
	vector<float> word_match_list;
	vector<float> char_match_list;
	vector<float> word_match_list_stopword;
	vector<float> char_match_list_stopword;
	vector<float> len_word_text1;
	vector<float> len_word_text2;
	
	vector<float> word_levenstein;
	vector<float> word_jaroWinkler;
	vector<float> word_tokenSetRatio;
	vector<float> word_partialRatio;
	vector<float> word_partialSortRatio;
	vector<float> word_sortRatio;
};

class DataFrame {
public:
    DataFrame();
	int get_rownum();
	cppjieba::Jieba* get_jieba();
    
	//train methods:
	TrainTemp read_csv(const char * filename);
	void compute_tfidf(TrainTemp train_temp);
	void compute_wordmatch(TrainTemp train_temp);
	void compute_len_feature(TrainTemp train_temp);
	void load_wordvec(const char * filename);
	void load_wordvec_norm(const char * filename);
	void compute_wordvec(TrainTemp train_temp);
	void compute_edit_distance(TrainTemp train_temp);
    void read_stopword(const char * filename);
	void build_weight();
	
	//predict methods:
	FeatureResult predict(string input1,string input2);
	void init_for_predict();
	PredictTemp fenci_for_predict(string input1,string input2);
	void compute_tfidf_for_predict(PredictTemp predict_temp,FeatureResult &feature_result_for_predict);
	void compute_wordvec_for_predict(PredictTemp predict_temp,FeatureResult &feature_result_for_predict);
	void compute_len_feature_for_predict(PredictTemp predict_temp,FeatureResult &feature_result_for_predict);
	void compute_wordmatch_for_predict(PredictTemp predict_temp,FeatureResult &feature_result_for_predict);
	void compute_edit_distance_for_predict(PredictTemp predict_temp,FeatureResult &feature_result);
	
	//share only one:
	cppjieba::Jieba* j;
	map<string,VectorXf> word2vec;
	map<string,VectorXf> word2vec_norm;
	map<string,int> word2count;// TODO OOV handle
	map<string,int> char2count;
	vector<bool> stopword_vector;
    vector<float> weight;
	map<string, float> idf_word;
	map<string, int> word2id; // word2id + char2id // TODO OOV handle
	vector<string> id2word;

    vector<string> text1_column;
    vector<string> text2_column;
    vector<int> score_column;
	vector<int> label_column;
	
	
	FeatureResult feature_result;
};

}